#include"PageLibPreprocessor.h"
#include"Configuration.h"
#include"SimHash.h"
#include"tinyxml2.h"
#include"SloSplitTool.h"

#include<iostream>
#include<fstream>
#include<sstream>
#include <regex>
#include <thread>
#include <chrono>
#include <list>
#include<set>
#include <cmath>
using std::stringstream;
using std::istringstream;
using std::ifstream;
using std::cout;
using std::ofstream;
using std::list;
using std::set;
using namespace tinyxml2;
using namespace simhash;
PageLibPreprocessor::PageLibPreprocessor(Configuration&conf){
    //读配置文件拿到网页库和网页偏移库路径
    map<string,string> myconf=conf.getConfigMap();
    string off_filename=myconf["offset"];
    string page_filename=myconf["pagelib"];
    std::cout<<"filename:"<<myconf["offset"]<<"\n";
    std::cout<<"filename:"<<myconf["pagelib"]<<"\n";
    //开始工作
    doProcess(off_filename,page_filename);
}
void PageLibPreprocessor::readInfoFromFile(string filename){
    ifstream ifs(filename);
    stringstream buf;
    buf<<ifs.rdbuf();
    string line;
    int count=0;
    while(getline(buf,line)){
        count++;
        int id,off,length;
        istringstream iss(line);
        iss>>id>>off>>length;
        _offsetLib.insert({id,{off,length}});
    }

}
void PageLibPreprocessor::cutRedundantPages(string filename){
    ifstream ifs(filename);
    uint64_t fingerprint;
    vector<pair<int,uint64_t>> _findex;
    for(size_t idx=1;idx<=_offsetLib.size();idx++){
        size_t off=_offsetLib[idx].first;
        size_t length=_offsetLib[idx].second;
        string buf;
        buf.resize(length+1);
        ifs.seekg(off);
        ifs.read(&buf[0],length);
        //解析拿到文章内容
        string afterRss=rssParse(buf);
        //计算指纹
        if(afterRss!=string()){
        fingerprint=simHash(afterRss);
            if(_findex.size()==0){
                _findex.push_back({idx,fingerprint});
            }else{
                auto it=_findex.begin();
                for(;it!=_findex.end();++it){
                    if(Simhasher::isEqual(it->second, fingerprint)){
                        break;
                    }
                }
                if(it==_findex.end()){
                    _findex.push_back({idx,fingerprint});
                }
            }
        }
        
    }
    //循环结束
    ofstream ofs("/home/zzh/cpp/project/v1_project/search/data/newpagelib.dat");
    map<int,pair<int,int>> newIndex;
    int noff=0;
    for(size_t idx=1;idx<=_findex.size();++idx){
        size_t off=_offsetLib[_findex[idx].first].first;
        size_t length=_offsetLib[_findex[idx].first].second;
        string buf;
        buf.resize(length);
        ifs.seekg(off);
        ifs.read(&buf[0],length);
        buf=replaceDocId(buf,to_string(idx));
        newIndex.insert({idx,{noff,buf.size()}});
        noff+=buf.size();
        ofs<<buf;
    }
    
    ofstream offs("/home/zzh/cpp/project/v1_project/search/data/newoffset.dat");
    for(auto ele:newIndex){
        offs<<ele.first<<" "<<ele.second.first<<" "<<ele.second.second<<"\n";
    }
    
}
uint64_t PageLibPreprocessor::simHash(string content){
    size_t topN = 5;
    uint64_t u64 = 0;
    vector<pair<string ,double> > res;
    SimHash::getInstance()->getSimhasher().extract(content, res, topN);
    SimHash::getInstance()->getSimhasher().make(content, topN, u64);
    return u64;
}
string PageLibPreprocessor::rssParse(string content){
    tinyxml2::XMLDocument doc;
    doc.Parse(content.c_str());
    tinyxml2::XMLElement* root = doc.FirstChildElement("doc");
    if (!root) {
        return string();
    }
    string Rsscontent;
    if( root->FirstChildElement("description")){
        Rsscontent=root->FirstChildElement("description")->GetText();

    }else if(root->FirstChildElement("content:encoded")){
        Rsscontent=root->FirstChildElement("content:encoded")->GetText();
    }else{
        Rsscontent=string();
    }
    std::regex reg("<[^>]+>");//通用正则表达式
    if(Rsscontent!=string()){
        Rsscontent = regex_replace(Rsscontent, reg, "");
    }
    return Rsscontent;

}
void PageLibPreprocessor::buildInvertindexTable(){
    //读文件索引
    ifstream ifs("/home/zzh/cpp/project/v1_project/search/data/newoffset.dat");
    stringstream buf;
    buf<<ifs.rdbuf();
    unordered_map<int, std::pair<int, int>> newOffset;
    string line;
    int count=0;
    while(getline(buf,line)){
        count++;
        int id,off,length;
        istringstream iss(line);
        iss>>id>>off>>length;
        newOffset.insert({id,{off,length}});
    }
    ifs.close();

    //导入停用词
    vector<string> stopfile;
    stopfile.push_back("/home/zzh/cpp/project/v1_project/yuliao/stop_words_zh.txt");
    stopfile.push_back("/home/zzh/cpp/project/v1_project/yuliao/stop_words_eng.txt");
    unordered_map<string,int> stopMap;
    for(auto ele:stopfile){
    ifstream ifs(ele);
        if(ifs.is_open()){
            cout<<"file open success!\n";
        }else{
            perror("file open fail\n");
            return;
        }
        //读入整个文本文件
        stringstream buf;
        buf<<ifs.rdbuf();
        string content=buf.str();
        istringstream iss(content);
        string word;
        while(iss>>word){
            ++stopMap[word];
        }
    }
        
    //根据索引读文章
    ifs.open("/home/zzh/cpp/project/v1_project/search/data/newpagelib.dat");
    unordered_map<string,set<pair<int,double>>> invert;
    map<string,int> allPageWords;
    for(size_t idx=1;idx<=newOffset.size();idx++){
        set<string> onePageWord;
        size_t off=newOffset[idx].first;
        size_t length=newOffset[idx].second;
        string buf; 
        buf.resize(length);
        ifs.seekg(off);
        ifs.read(&buf[0],length);
        //读出xml格式中的文章正文内容
        buf=rssParse(buf);
        //开始算权重
        if(buf!=string()){
            vector<string> words=SloSplitTool::getInstance()->cut(buf);
            onePageWord.insert(words.begin(),words.end());
            
            for(auto ele:onePageWord){
                if(stopMap.count(ele)==0){//去停用词
                ++allPageWords[ele];
                }
            }
        }
    }
    for(size_t idx=1;idx<=newOffset.size();idx++){
        map<string,int> oneMPageWords;
        list<pair<string,double>> powerList;
        size_t off=newOffset[idx].first;
        size_t length=newOffset[idx].second;
        string buf; 
        buf.resize(length);
        ifs.seekg(off);
        ifs.read(&buf[0],length);
        //提取正文内容
        buf=rssParse(buf);
        if(buf!=string()){
            //分词
            vector<string> words=SloSplitTool::getInstance()->cut(buf);
            //词频统计
            for(auto ele:words){
                if(stopMap.count(ele)==0){
                ++oneMPageWords[ele];
                }
            }

            for(auto ele:oneMPageWords){
                //算一篇文章中某个词的权重
                double TF=(double)ele.second;
                double DF=(double)(allPageWords[ele.first]);
                double IDF=log2(3646.0/(DF+1.0));
                double power=TF*IDF;
                //统计一篇文章中所有词以及其对应的权重
                powerList.push_back({ele.first,power});
                // invert[ele.first].insert({idx,power});
            }
            double totalPower=0;
            for(auto ele:powerList){
                totalPower+=pow(ele.second,2);
            }
            double sqrtPower=sqrt(totalPower);
            for(auto ele:powerList){
                double geometryPower=ele.second/sqrtPower;
                invert[ele.first].insert({idx,geometryPower});
            }

        }
    }
    ofstream ofs("/home/zzh/cpp/project/v1_project/search/data/invertIndex.dat");
    for(auto ele:invert){
        ofs<<ele.first<<" ";
        for(auto sele:ele.second){
            ofs<<sele.first<<" "<<sele.second<<" ";
        }
        ofs<<"\n";
    }
    
}
//void PageLibPreprocessor::storeOnDisk(){}
void PageLibPreprocessor::doProcess(string &off_filename,string &page_filename){
    //将偏移库读入_offsetlib中
    readInfoFromFile(off_filename);
    //去重
    cutRedundantPages(page_filename);
    //建立倒转索引
    buildInvertindexTable();
}

string PageLibPreprocessor::replaceDocId(const string& content, const string& newDocId) {
    std::string startTag = "<docid>";
    std::string endTag = "</docid>";
    size_t startPos = content.find(startTag);
    size_t endPos = content.find(endTag, startPos + startTag.length());

    if (startPos != std::string::npos && endPos != std::string::npos) {
        // 获取旧的 docid
        std::string oldDocId = content.substr(startPos + startTag.length(), endPos - (startPos + startTag.length()));
        // 替换新的 docid
        std::string newContent = content.substr(0, startPos + startTag.length()) + newDocId + content.substr(endPos);
        return newContent;
    } else {
        return content;
    }
}
